# encoding: utf-8
import sys
reload(sys)
sys.setdefaultencoding('utf8')
__author__ = 'wwj'
import MySQLdb
import MySQLdb.cursors
from datetime import *
sys.path.append("/Users/wwj/Downloads/shangtianjun/girl/girl/dbShell/")
sys.path.append( "/var/crawler/spider/crawler/girl/dbShell/")
from dbshell import dbshell
import copy
import re
from zlib import crc32
# from ..dbshell import dbshell
class main(dbshell):
    orderItem = [
        ['gid', ''],
        ['sid', ''],
        ['title', ''],
        ['thumb', ''],
        ['url', ''],
    ]
    existgirl = {}
    mongoTable = "diary"
    mysqlTable = "diary"
    def __init__(self, spidername):
        super(main, self).__init__(spidername)
        # self.geturltool("select distinct(url) from diary")
        # self.mongofrom = self.mongo.find(self.where,no_cursor_timeout=True).batch_size(self.exesize).sort("_id",-1).skip(1000000)
        self.getgirls()

    def getgirls(self):
        # tmp = self.client[self.config.get('mongoDb')]["exitgirl"].find({}, {"_id": 0, "url": 1})
        # if tmp.count() == 0:
            # tmp = self.getdb("select url,id,sid from girl")
            # self.client[self.config.get('mongoDb')]["exitgirl"].insert_many(
            #     [dict(url={i.split()[0]: (i.split()[1], i.split()[2])}) for i in tmp])
            # tmp = self.client[self.config.get('mongoDb')]["exitgirl"].find({}, {"url": 1, "_id": 0})
        # for x in tmp:
        #     self.existgirl.update(x["url"])
        # tmp = self.getdb("select url,id,sid from girl")
        # self.redisconn.has
        if not self.redisconn.exists("girlss"):
            sql = "select url,id,sid from girl"
            tmp = self.getdb(sql)
            for x in tmp:
                self.redisconn.hset("girlss",x["url"],(x["id"],x["sid"]))


    def clean(self, y):
        data = self.data
        if not y.get("girlUrl"):
            raise self.myException("no girlurl")
        url = y["girlUrl"].replace("https://www.cityheaven.net", "")
        if self.redisconn.hget("girlss",url):
            data["gid"],data["sid"] = eval(self.redisconn.hget("girlss",url))
        else:
            print "ccc"
            return
        data["title"] = y["title"]
        data["name"] = y["name"]
        if y.get("image_urls"):
            data["thumb"] = re.sub("\?cache.*", "", y.get("image_urls"))
        else:
            data["thumb"] = ""
        data["time"] = y.get("time")
        data["context"] = y.get("context")
        data["url"] = y["url"].replace("https://www.cityheaven.net", "")
        data["urlHash"] = str(crc32(data["url"])).replace("-","")
        tmpdata = copy.deepcopy(data)
        self.datas.append(tmpdata)
        # print data


if __name__ == '__main__':
    obj = main("city")
    obj()
